{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 首先 import 必要的模块\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 读取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pregnants</th>\n",
       "      <th>Plasma_glucose_concentration</th>\n",
       "      <th>blood_pressure</th>\n",
       "      <th>Triceps_skin_fold_thickness</th>\n",
       "      <th>serum_insulin</th>\n",
       "      <th>BMI</th>\n",
       "      <th>Diabetes_pedigree_function</th>\n",
       "      <th>Age</th>\n",
       "      <th>Target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.639947</td>\n",
       "      <td>0.866045</td>\n",
       "      <td>-0.031990</td>\n",
       "      <td>0.670643</td>\n",
       "      <td>-0.181541</td>\n",
       "      <td>0.166619</td>\n",
       "      <td>0.468492</td>\n",
       "      <td>1.425995</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.844885</td>\n",
       "      <td>-1.205066</td>\n",
       "      <td>-0.528319</td>\n",
       "      <td>-0.012301</td>\n",
       "      <td>-0.181541</td>\n",
       "      <td>-0.852200</td>\n",
       "      <td>-0.365061</td>\n",
       "      <td>-0.190672</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.233880</td>\n",
       "      <td>2.016662</td>\n",
       "      <td>-0.693761</td>\n",
       "      <td>-0.012301</td>\n",
       "      <td>-0.181541</td>\n",
       "      <td>-1.332500</td>\n",
       "      <td>0.604397</td>\n",
       "      <td>-0.105584</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.844885</td>\n",
       "      <td>-1.073567</td>\n",
       "      <td>-0.528319</td>\n",
       "      <td>-0.695245</td>\n",
       "      <td>-0.540642</td>\n",
       "      <td>-0.633881</td>\n",
       "      <td>-0.920763</td>\n",
       "      <td>-1.041549</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-1.141852</td>\n",
       "      <td>0.504422</td>\n",
       "      <td>-2.679076</td>\n",
       "      <td>0.670643</td>\n",
       "      <td>0.316566</td>\n",
       "      <td>1.549303</td>\n",
       "      <td>5.484909</td>\n",
       "      <td>-0.020496</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   pregnants  Plasma_glucose_concentration  blood_pressure  \\\n",
       "0   0.639947                      0.866045       -0.031990   \n",
       "1  -0.844885                     -1.205066       -0.528319   \n",
       "2   1.233880                      2.016662       -0.693761   \n",
       "3  -0.844885                     -1.073567       -0.528319   \n",
       "4  -1.141852                      0.504422       -2.679076   \n",
       "\n",
       "   Triceps_skin_fold_thickness  serum_insulin       BMI  \\\n",
       "0                     0.670643      -0.181541  0.166619   \n",
       "1                    -0.012301      -0.181541 -0.852200   \n",
       "2                    -0.012301      -0.181541 -1.332500   \n",
       "3                    -0.695245      -0.540642 -0.633881   \n",
       "4                     0.670643       0.316566  1.549303   \n",
       "\n",
       "   Diabetes_pedigree_function       Age  Target  \n",
       "0                    0.468492  1.425995       1  \n",
       "1                   -0.365061 -0.190672       0  \n",
       "2                    0.604397 -0.105584       1  \n",
       "3                   -0.920763 -1.041549       0  \n",
       "4                    5.484909 -0.020496       1  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv(\"FE_pima-indians-diabetes.csv\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 保存特征名字以备后用（可视化）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['pregnants', 'Plasma_glucose_concentration', 'blood_pressure',\n",
      "       'Triceps_skin_fold_thickness', 'serum_insulin', 'BMI',\n",
      "       'Diabetes_pedigree_function', 'Age', 'Target'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "feat_names = train.columns \n",
    "print(feat_names)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 准备数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = train['Target']   \n",
    "X = train.drop([\"pregnants\", \"Target\"], axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 默认参数的Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "lr = LogisticRegression()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 正确率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logloss of each fold is:  [0.78571429 0.74675325 0.76623377 0.81045752 0.74509804]\n",
      "cv logloss is: 0.7708513708513708\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "loss = cross_val_score(lr, X, y, cv=5, scoring='accuracy')\n",
    "#%timeit loss_sparse = cross_val_score(lr, X_train_sparse, y_train, cv=3, scoring='neg_log_loss')\n",
    "print ('logloss of each fold is: ',loss)\n",
    "print ('cv logloss is:', (loss).mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 采用5折交叉验证和log似然损失"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 14 candidates, totalling 70 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Done  10 tasks      | elapsed:    2.3s\n",
      "[Parallel(n_jobs=4)]: Done  70 out of  70 | elapsed:    2.5s finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise',\n",
       "       estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False),\n",
       "       fit_params=None, iid=True, n_jobs=4,\n",
       "       param_grid={'penalty': ['l1', 'l2'], 'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=5)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#需要调优的参数\n",
    "penaltys = ['l1','l2']\n",
    "Cs = [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "tuned_parameters = dict(penalty = penaltys, C = Cs)\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "grid= GridSearchCV(lr, tuned_parameters, cv=5, scoring='neg_log_loss',n_jobs=4,verbose=5)\n",
    "grid.fit(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.48483658874027397\n",
      "{'C': 1, 'penalty': 'l1'}\n"
     ]
    }
   ],
   "source": [
    "print(-grid.best_score_)\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Users\\Administrator\\Anaconda3-5.3.1\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\Users\\Administrator\\Anaconda3-5.3.1\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3Xl4VPXZ//H3nZ1A2MNOEkRQFlkkAsENqwIudaniDqitaKu2ta2tdtGKWu3T5+njry2Pa1WkKiLWSt2gKq6sAUFZRBFBIooIIqAsWe7fH3PQGAIzWYaTmXxe1zVXZs6cM/M5grk553zP9zZ3R0REZH9Swg4gIiINn4qFiIhEpWIhIiJRqViIiEhUKhYiIhKVioWIiESlYiEiIlGpWIiISFQqFiIiElVa2AHqS9u2bb2goCDsGCIiCWXhwoWfuXtutPXiWizMbBTw/4BU4D53v73K+/8LHBe8zAbauXvL4L1xwG+D925x90n7+66CggKKi4vrM76ISNIzs7WxrBe3YmFmqcBE4ESgBFhgZtPdffmeddz9mkrrXw0MDJ63Bm4ECgEHFgbbfh6vvCIism/xvGYxGFjl7qvdfTcwBTh9P+ufDzwaPB8J/MfdNwcF4j/AqDhmFRGR/YhnsegMrKv0uiRYthczywe6AS/VdFsREYm/eF6zsGqW7Ws+9POAae5eXpNtzWw8MB4gLy+vNhlFJEmVlpZSUlLCzp07w47SIGRlZdGlSxfS09NrtX08i0UJ0LXS6y7A+n2sex5wZZVth1fZ9uWqG7n7PcA9AIWFhWrMISJfKykpIScnh4KCAsyq+/dn4+HubNq0iZKSErp161arz4jnaagFQA8z62ZmGUQKwvSqK5nZIUArYE6lxTOAEWbWysxaASOCZSIiMdm5cydt2rRp9IUCwMxo06ZNnY6y4nZk4e5lZnYVkV/yqcD97r7MzCYAxe6+p3CcD0zxSi373H2zmd1MpOAATHD3zfHKKiLJSYXiG3X9bxHX+yzc/Vng2SrLbqjy+vf72PZ+4P64hQuUVzh/fP4dxgzNp2vr7Hh/nYg0YOfeHTnB8djlRSEnaXga/XQfazd9yZT5H3L2XbNZ+cm2sOOISBJp1qzZ189HjRpFy5YtOfXUU6td98orr2TAgAH07t2bJk2aMGDAAAYMGMC0adNq9J2LFi3i+eefr1Pu6jT6YnFQbjOmXhH5V8Tou2ZTvEZnu0Sk/l177bVMnjx5n+9PnDiRxYsX8+yzz9K9e3cWL17M4sWLOfvss2v0PSoWcXRoh+ZMu2IYbZplcuF983hxxYawI4lIkjn++OPJycmp1bbvvfceI0eOZNCgQRxzzDG8++67AEyZMoW+ffvSv39/jjvuOHbs2MGECRN4+OGHa3VUsj9JM5FgXXVtnc3jVxRxyQMLGD95IX88qx9nD+oSdiwRqQc3/XsZy9dvjbre8o8j6+y5drE/vTs158bv9qlztliMHz+e++67j+7du/PGG29w1VVXMXPmTG666SZefvll2rdvz5YtW2jSpAk33HADS5cu5Y477qjXDCoWlbRtlsmj44dyxeSF/OLxJWzavovLj+0ediwRacS2bNnC3LlzOeuss75eVlZWBsCRRx7J2LFjGT16NN/73vfimkPFoopmmWn8/eJCfjZ1Cbc99w6bvtzNdaMOJSVFQ/BEElWsRwANcTSUu9O2bVsWL16813v33nsv8+bN4+mnn6Z///689dZbccuhaxbVyExL5S/nDWRsUT73vLqaa6e9RWl5RdixRKQRatWqFR07duTJJ58EoKKigiVLlgCwevVqhg4dys0330yrVq346KOPyMnJYdu2+h/ZqWKxD6kpxk2n9eGaE3ryxKISLp+8kB27y6NvKCJSjaOPPprRo0fz4osv0qVLF2bMiH1SiilTpnDXXXfRv39/+vTpw9NPPw3ANddcw2GHHcZhhx3GCSecQN++ffnOd77DkiVLGDhwYL1e4LZKN04ntMLCQo9X86N/zF3L755ayuF5rbh/3BG0yK7dRFwicuCsWLGCXr161Wibhngaqj5V99/EzBa6e2G0bXXNIgYXDc2nddMMfjplMaPvns1Dlw6hQ4ussGOJSD1L1iJRH3QaKkYnH9aRBy85gvVbdnLWnbN5f+P2sCOJiBwwKhY1MOzgtkwZP5SdpeWMvmsOS9ZtCTuSiMgBoWJRQ307t2DaD4eRnZHK+ffO5bX3NoYdSUQk7lQsaqFb26b884fDyGudzaUPLmD6kn31dBIRSQ4qFrXUrnkWj11exMCurfjJlDeZNHtN2JFEpK4eOCXykL2oWNRBiybpPPT9wZzQqz03Tl/Gn2euJFmGIotI3e2Zonzx4sUUFRXRp08f+vXrx2OPPbbXug19inINna2jrPRU7rzwcH795Nv85aVVbNy+m1vO6EuqpgcRkUB2djYPPfQQPXr0YP369QwaNIiRI0fSsmXLr9eZOHEiAGvWrOHUU0+tdnqPWCxatIilS5cyatSoesm+h44s6kFaagp/PKsfPxrenUfnf8iVDy9iZ6nu9haRiJ49e9KjRw8AOnXqRLt27di4MfbBMZqiPImYGb8cdShtmmVy89PLufiB+dw7tpCcLN3tLRK6566DT96Ovt4nwUR8sVy36HAYnHR7jaPMnz+f3bt307177DNaa4ryJPT9o7rRpmkGv3h8CefdM5cHLxlMbk5m2LFEpAH4+OOPGTNmDJMmTSIlJbYTO5qiPImdMbAzLbPT+eE/FnH2XbOZfOkQ8tpkhx1LpPGK9QhgzxHFJc/Ue4StW7dyyimncMsttzB06NCYt9MU5Ulu+CHtePiyIXyxo5Sz7podU5cuEUlOu3fv5swzz/z6KKAmNEV5I3B4XiumXVFEWopx7t1zmLt6U9iRRCQEU6dO5dVXX+XBBx/8ekhsTUY7aYryehTPKcrrav2WHYz5+zzWfb6Dv54/kJF9OoQdSSTp1WaK8niehmoI6jJFuY4sDoBOLZsw7Yph9O7YnB/+YyFT5n8YdiQRqc4lzyRtoagrFYsDpFXTDB65bAhH98jlun++zcRZq3S3t4gkDBWLAyg7I437xhVyxoBO/GnGSiY8vZyKChUMkXjRP8i+Udf/Fho6e4Clp6bw53MG0LppJve/8QGbv9zNn87uT0aa6rZIfcrKymLTpk20adMGs8Y9/Y67s2nTJrKyat/hU8UiBCkpxu9O7UXbnAz+6/mVfP5VKXdddDjZGfrjEKkvXbp0oaSkpEbTaiSzrKwsunTpUuvt9dspJGbGj4YfTJumGVz/z7e54N55PHDxEbRqmhF2NJGkkJ6eTrdu3cKOkTR07iNk5x6Rx50XDWL5x1s5+67ZfLRlR9iRRET2omLRAIzs04HJlw7m0227OPvO2by3of7vvhQRqYu4FgszG2VmK81slZldt491zjGz5Wa2zMweqbS83MwWB4/p8czZEAw5qA2PjS+irMIZffccFq79POxIIiJfi1uxMLNUYCJwEtAbON/MeldZpwdwPXCku/cBflrp7R3uPiB4nBavnA1J707NeeKKYbRsks6F981l1spPw44kIgLE98hiMLDK3Ve7+25gCnB6lXUuAya6++cA7t7ofzvmtcnm8SuG0T23GZdNKubJN0vCjiQiEtdi0RlYV+l1SbCssp5ATzN7w8zmmlnlPoBZZlYcLD+jui8ws/HBOsXJNDwuNyeTKeOHckRBa655bAn3vbY67Egi0sjFs1hUdxdM1VsI04AewHDgfOA+M9vTlDYvmNzqAuAOM9urrZS73+Puhe5emJubW3/JG4CcrHQeuOQITurbgVueWcHtz72ju1FFJDTxLBYlQNdKr7sA66tZ5yl3L3X3D4CVRIoH7r4++LkaeBkYGMesDVJWeip/u+BwLhySx12vvM+vnniLsvKKsGOJSCMUz2KxAOhhZt3MLAM4D6g6qulfwHEAZtaWyGmp1WbWyswyKy0/Elgex6wNVmqKccsZffnJ8T2YWlzCFf9YxM7S8rBjiUgjE7di4e5lwFXADGAFMNXdl5nZBDPbM7ppBrDJzJYDs4Br3X0T0AsoNrMlwfLb3b1RFguI3O19zYk9mXB6H158ZwNj/z6fL3aUhh1LRBoRNT9KME+/tZ5rHltM99xmPHTpYNo1r/3EYCIian6UpE7t14kHLh7Mh5u/4nt3zuaDz74MO5KINAIqFgnoqB5tmTJ+KF/tLufsO2fzdskXYUcSkSSnYpGg+nVpybQrishKT+W8e+Ywe9VnnHv3HM69e07Y0UQkCalYJLCDcpvxzx8No0urbC5+YAGbvtwddiQRSVIqFgmuffMspl5eRL8uLVj16XY2bN0ZdiQRSUIqFgAPnBJ5JKgW2elM/v4QWjZJZ82mr3QNQ0TqnYpFkmiSkUr3ds1IMZg0Z03YcUQkyahYJJG0FKNts0ymL1nPZl2/EJF6pGKxaxt8vgZKk6Odafvmmewuq+CxBeuirywiEiMVi13bYdv6SMFIcI9dXsS/rz6aooPa8I+5aymvSI6780UkfCoWzTtCi66wYxN88GrYaerFuGH5fLRlBy+90+h7SYlIPVGxAMjpBKmZ8Pz1UJH4M7qe0Ks9HVtk8dCcNWFHEZEkoWIBkJIKrbrBhqWw6KGw09RZWmoKFw7J47X3PuP9jdvDjiMiSUDFYo/sNpA3DF66BXYm/n0K5w3OIyM1hclz1oYdRUSSgIoFwCXPwKXPwqjb4KtN8Mp/hZ2ozto2y+SUfh15YmEJ23eVhR1HRBKcikVlnQbAwAth3t2w6f2w09TZmKJ8tu0q48k3Pwo7iogkOBWLqr5zA6RlwYzfhJ2kzgZ2bclhnVvw0Ow1JEuTKxEJh4pFVTnt4Zifw7vPwfsvhZ2mTsyMsUX5vPfpduau3hx2HBFJYCoW1Rn6I2hVAM//GsoT+3z/d/t3omV2uobRikidqFhUJy0TRtwCG1fAwgfCTlMnWempnHtEV2Yu38DHXyTHlCYicuCpWOzLoadCwdEw6w+w4/Ow09TJRUPyqXDnkXkfhh1FRBKUisW+mEWG0u7cAi//Mew0ddK1dTbHH9qeR+d/yK6yxL9DXUQOPBWL/elwGBw+FhbcCxvfDTtNnYwtyuez7bt57u1Pwo4iIglIxSKa434L6dkwM7GH0h51cFsOattUjZFEpFZULKJplgvH/hLemwnvvRB2mlpLSTHGFOXz5odb1HZVRGpMxSIWgy+H1gfBjOuhvDTsNLV21qAuZGekahitiNSYikUs0jJgxK3w2buw4O9hp6m15lnpnDmwM08tWc/narsqIjWgYhGrQ06Cg4bDy7fBV4l7N/TYooJI29VitV0VkdipWMTKDEbeBru2RgpGgjqkQw5DD2qttqsiUiMqFjXRvjcUXho5FfXpirDT1NrYogJKPt/BLLVdFZEYqVjU1PBfQ2YzmPFrSNCZXE/s3Z4OzbM0jFZEYhbXYmFmo8xspZmtMrPr9rHOOWa23MyWmdkjlZaPM7P3gse4eOaskaZt4NjrIjPSvjsj7DS1kl6p7epqtV0VkRjErViYWSowETgJ6A2cb2a9q6zTA7geONLd+wA/DZa3Bm4EhgCDgRvNrFW8stbY4MugTY/IjXpliTmq6LzBeaSnGpPnqu2qiEQXzyOLwcAqd1/t7ruBKcDpVda5DJjo7p8DuPuek+gjgf+4++bgvf8Ao+KYtWZS02HkH2DTqshUIAkoNyeTkw/ryLTiEr5U21URiSKexaIzUHl8ZkmwrLKeQE8ze8PM5prZqBpsG66eI+DgEyKTDH75WdhpamVsUYHaropITOJZLKyaZVWvCKcBPYDhwPnAfWbWMsZtMbPxZlZsZsUbN26sY9xaGPkH2L0dZt164L+7Hhye15K+nZszec5atV0Vkf2KZ7EoAbpWet0FWF/NOk+5e6m7fwCsJFI8YtkWd7/H3QvdvTA3N7dew8ck9xA44gew8EH4ZOmB//46MjPGDi1g5YZtzPsgcW80FJH4i2exWAD0MLNuZpYBnAdMr7LOv4DjAMysLZHTUquBGcAIM2sVXNgeESxreIZfB1ktEnYo7WkD1HZVRKKLW7Fw9zLgKiK/5FcAU919mZlNMLPTgtVmAJvMbDkwC7jW3Te5+2bgZiIFZwEwIVjW8GS3jtx78cErsPLZsNPUWFZ6KucWdmXGMrVdFZF9s2Q5V11YWOjFxcXhfHl5Gdx1JJTtgivnRXp4J5B1m7/imD/N4urjDuZnIw4JO46IHEBmttDdC6Otpzu460NqWuRi9+cfwLy7wk5TY11bZ/OdQ9rxiNquisg+qFjUl4OPhx4j4ZU/wfbEm3Np7LACPtu+m+eXqu2qiOxNxaI+jbwVynbAS7eEnaTGjj64Ld3aNmXS7DVhRxGRBqjGxcLMUsyseTzCJLy2PSJd9RY9BB+/FXaaGklJMcYMzWfRh1tY+pHarorIt8VULMzsETNrbmZNgeXASjO7Nr7REtSxv4yMkHr++oQbSnvWoC40SVfbVRHZW6xHFr3dfStwBvAskAeMiVuqRNakJRz3G1j7OqyoeltJw9aiSTpnHt6Zpxar7aqIfFusxSLdzNKJFIun3L2UaqbfkMDh46Bdb5j5WyjdGXaaGhlblM+usgoeX6i2qyLyjViLxd3AGqAp8KqZ5QNb4xUq4aWmwajbYMuHMPf/wk5TI4d2aM7gbq2ZrLarIlJJTMXC3f/i7p3d/WSPWEswTYfsw0HD4ZBT4LX/gW2JNRx1XFEB6zbv4OWViTcEWETiI9YL3D8JLnCbmf3dzBYB34lztsQ34ubIXd0v3hx2khoZ0ac97ZtnMmmOGiOJSESsp6EuDS5wjwBygUuA2+OWKlm06Q5DfwiLH4b1b4adJmaRtqv5vPruRj747Muw44hIAxBrsdjTX+Jk4AF3X0L1PSekqmN+Adlt4LnrEmoo7XmDu0baruroQkSIvVgsNLOZRIrFDDPLASriFyuJZLWA438H6+bCsifDThOzdjlZnNS3I48vXKe2qyISc7H4PnAdcIS7fwVkEDkVJbEYOAbaHwb/uQFKE2ca8HHD8tm2s4x/LVbbVZHGLtbRUBVEutX91sz+Gxjm7ok1n0WYUlLhpNvhi3Uw+29hp4nZ4Xmt6N1RbVdFJPbRULcDPyEy1cdy4Mdmdls8gyWdgqOg12nw+p9h614dYhskM2PcsHze+WQb89V2VaRRi/U01MnAie5+v7vfD4wCTolfrCR14gSoKIMXbgo7ScxO69+ZFk3SeUgXukUatZrMOtuy0vMW9R2kUWjdDYquhLemQMnCsNPEpElGKuce0ZUZyz7hky8Sa+oSEak/sRaL24A3zexBM5sELAT+EL9YSezon0Oz9vB84gylvWhIPuXuPDL/w7CjiEhIYr3A/SgwFPhn8Chy9ynxDJa0MnPg+BugZD68PS3sNDHJa5PNcYe045F5H7K7TCOmRRqj/RYLMzt8zwPoCJQA64BOwTKpjf4XQMf+8MKNsDsx7pAeW5TPZ9t38dzSj8OOIiIhSIvy/v/s5z1H80PVTkoKjLodHjgJ3vgLHHd92ImiOqZHLgVtspk8Zy2nD+gcdhwROcD2WyzcXTPLxkv+MOhzJrzx/+DwMdCiS9iJ9islxbhoaD63PLOCpR99Qd/OGuMg0pjEep/F96p5HG9m7eIdMKmdOAFweOH3YSeJyehBXWmSnqr5okQaoZpM93EfcGHwuBf4GfCGmam9am21zINhV8Pbj8O6+WGniapFdjpnDOzMvxZ/xJav1HZVpDGJtVhUAL3c/Sx3PwvoDewChgC/ile4RuHIn0JOR3juV1DR8Ecafd12tbgk7CgicgDFWiwK3H1DpdefAj3dfTNQWv+xGpHMZnD8jbB+Ebz1WNhpourVsTmDC9R2VaSxibVYvGZmT5vZODMbB0wn0ou7KbAlfvEaiX7nQudB8OJNsGt72GmiGjssnw83f8Ur76rtqkhjEWuxuBJ4ABgADAQmAVe6+5caMVUP9gyl3fYxvHFH2GmiGtmnA+1yMpk0Wxe6RRqLWO/gduB14CXgBeBV15zV9avrYDhsNMz+K2xp2NNqpKemcMGQPF55dyNr1HZVpFGIdejsOcB84GzgHGCemZ0dz2CN0gm/ByzSJKmBu2BwHmkpxuS5OroQaQxiPQ31GyJd8sa5+1hgMPC7aBuZ2SgzW2lmq8zsumrev9jMNprZ4uDxg0rvlVdaPj3WHUpoLbrAkT+JtF9dOzvsNPvVrnkWJx3WkanF6/hqt9quiiS7WItFirtXvpq5Kdq2ZpYKTAROIjLU9nwz613Nqo+5+4DgcV+l5TsqLT8txpyJ78ifQPPOkVlpG/hQ2rFFkbarTy1OjGZOIlJ7sRaL581sRnAkcDHwDPBslG0GA6vcfbW77wamAKfXPmojkZENJ9wEHy+BJY+EnWa/CvNb0atjcybNXqO2qyJJLtYL3NcC9wD9gP7APe4e7Wa8zkRmqN2jJFhW1Vlm9paZTTOzrpWWZ5lZsZnNNbMzYsmZNA47G7oMhhcnwK5tYafZJzNjXFGk7eqCNZ+HHUdE4ijmTnnu/oS7/8zdr3H3J2PYxKr7mCqv/03khr9+REZZTar0Xp67FwIXAHeYWfe9vsBsfFBQijdu3BjjniQAs8hQ2u0b4LX9TfwbvtMHdKZ5VhqT5qwJO4qIxFG06w7bzGxrNY9tZrY1ymeXAJWPFLoA3zq57e6b3H1X8PJeYFCl99YHP1cDLxO5v4Mq29/j7oXuXpibmxslToLpMgj6nQdzJsLmD8JOs09NMlI5p7ArM5Z+woatarsqkqz2WyzcPcfdm1fzyHH35lE+ewHQw8y6mVkGcB6RO7+/ZmYdK708DVgRLG9lZpnB87bAkcDymu1aEjjhRkhJa/BDaS8aGrRdndew7w8RkdqL+TRUTbl7GXAVMINIEZjq7svMbIKZ7Rnd9GMzW2ZmS4AfAxcHy3sBxcHyWcDt7t74ikXzTnDUz2DFdFjzethp9qmgbVOG98zlkflquyqSrCxZRrEUFhZ6cXFx2DHqX+kO+NtgyGoBl78CKalhJ6rWrHc+5ZIHF/CX8wdyWv9OYccRkRiZ2cLg+vB+xe3IQupJehM48SbY8Da8OTnsNPt0bM9c8lpnM3nOmrCjiEgcqFgkgj5nQl4RvHgz7Pxi3+s9cErkEYKUFGNsUT4L1nzOsvX7ySgiCUnFIhGYwajb4KtN8Op/h51mn0YP6kpWeorarookIRWLRNFpIAy4EObeCZveDztNtVpkp3PGgEjb1S++Uk8skWSiYpFIjr8B0jJhZtQ5HEMzpiifnaUVPL5wXfSVRSRhqFgkkpz2cPTPYeUzsPrlsNNUq0+nFhxR0IqH5qylQm1XRZKGikWiGfojaJkPz18P5Q1zavCxRQVB29UkmoJFpJFTsUg06Vkw4mb4dDksmhR9/RCM7NOB3JxMHpqzJuwoIlJPVCwSUa/TIP8omHUr7NgSdpq9ZKSlcMHgPF5W21WRpKFikYi+Hkq7GV75r7DTVOuCIXmkmvEPtV0VSQoqFomqYz84fCzMvxs+ey/sNHtp3zyLkX07MLV4HTt2l4cdR0TqSMUikX3nt5DWBGb8Juwk1RpXVMDWnWU8tfijsKOISB2pWCSyZu3g2GvhvRmw6oWw0+zliIJWHNohh0lz1qrtqkiCU7FIdEOugFbdIkcXDewXspkxblgBKz7eSvFatV0VSWQqFokuLRNG3gob34FtH4edZi+nD+hETlYak2avCTuKiNSBikUyOORk6HYsfPEhlDesOZmyM9I4p7Arzy/9hE/VdlUkYalYJIM9Q2kryiIFo4EZMzSfsgrnkfkNL5uIxEbFIlm07wPNOkRORb1wE+zaHnairxW0bcqxPXN5ZN6HlJar7apIIlKxSCatCqBpLrz+Z/jrIFgyBSoaxi/nccPy+XTbLmYs+yTsKFIHy/5wFMv+cFTYMeosWfYDDty+qFgkk5Q0aHsIfP8FaNEZnrwc7h8BJQvDTsaxPduR1zqbh2Y3vju6k+kXkzReKhbJqOsRkYJxxp2w5UO47zvw5A9hW3j/qk9NMcYMzWf+ms0sX7816vrn3j2Hc++ecwCSiUgsVCySVUoKDLgArl4IR/4Ulk6LnJp6/X+hbFcokUYXdiEzLYXJc9dEXfeGTddyw6Zr4x9KRGKiYpHsMnPgxJvgR3Oh2zHwwu9h4hB459kDfhNfy+yMSNvVN9er7apIglGxaCzadIfzH4WL/gmpGTDlfPjH9+DTdw5ojDFF+ewoLVfbVZEEo2LR2Bx8PPzwDRj1R/hoIdw5DJ67DnYcmOk4+nZuwaD8Vkyeq7arIolExaIxSk2HoVfA1Ytg0LjINOd/HQTF90NF/KcTH1uUz9pNX/HKe2q7KpIoVCwas6Zt4dT/hfGvQO6h8PQ1cPexsOb1uH7tSX070rZZJpPnNL5htCKJSsVCIo2ULn4GRj8IO7fAg6fA1HGRYbdxkJGWwgVD8pi18lPWblLbVZFEoGIhEWbQ50y4cj4M/zW8OwP+dgTM+gPs/qrev+6CwXmkqO2qSMJQsZBvy8iG4b+Cq4vh0FPglT/C3wrh7Wn1OtS2Q4ssRvXpwNTiErVdFUkAKhZSvRZd4Oz74ZLnILsNPPF9eOAkWL+43r5ibFE+X+woZfoStV0VaehULGT/8ofB+Jfhu3+Bz96De4bD9Kthe91HMg3u1jrSdnW22q6KNHRxLRZmNsrMVprZKjO7rpr3LzazjWa2OHj8oNJ748zsveAxLp45JYqU1MgQ26sXQtGVsPiRyFDbOROhbHetP9bMGFOUz/KPt7JQbVdFGrS4FQszSwUmAicBvYHzzax3Nas+5u4Dgsd9wbatgRuBIcBg4EYzaxWvrBKjJi0jLVx/OCcyWeGMX0du6nvvhVp/5BkDOpOTlcZDGkYr0qDF88hiMLDK3Ve7+25gCnB6jNuOBP7j7pvd/XPgP8CoOOVMHpc8E3nEW25PuHAaXDAVvAIePgsePgc+W1Xjj2qamcboQV159u2P1XZVpAGLZ7HoDFSeAKgkWFbVWWb2lplNM7OuNdxWwmIGPUdGJig88WZYOxv+byjM/C3s/KJGHzWmKNJ29dH5mi9KpKGKZ7GwapZJG73dAAAMcElEQVRVvYr5b6DA3fsBLwCTarAtZjbezIrNrHjjRk0dEYq0DDjyx5HrGf3Phdl/i1zPWDQ55i593do25ZieuTw8b63aroo0UPEsFiVA10qvuwDrK6/g7pvcfU9zhXuBQbFuG2x/j7sXunthbm5uvQWXWshpD6dPhMtegtYHwfSr4N7j4MN5MW0+rijSdnXmsg1xDioitRHPYrEA6GFm3cwsAzgPmF55BTPrWOnlacCK4PkMYISZtQoubI8IlklD1/lwuHQGfO8+2P5ppK3rEz+AL/Z/L8XwQ9rRtXUTJs1Zc0BiikjNxK1YuHsZcBWRX/IrgKnuvszMJpjZacFqPzazZWa2BPgxcHGw7WbgZiIFZwEwIVgmicAM+o2GqxbA0b+A5dMjd4G/8ico3VHtJqkpxkVD8pn/wWbe+SR621URObDiep+Fuz/r7j3dvbu73xosu8HdpwfPr3f3Pu7e392Pc/d3Km17v7sfHDweiGdOiZPMZnD87+Cq+XDwCTDrFpg4GJY/Ve3UIecUdiUzLUXDaEUaIN3BLfHXqgDOnQxjp0NGM5g6FiZ9Fz5Z+u3VmmZw+oBOPLnoI7Z7ZjhZRaRaKhZy4Bx0LFz+Gpz837BhKdx9NDzzc/jqmzOMY4sK2FFazgu7+4UYVESqUrGQAys1DQZfFunSd8QPoPgB+MtAmHcPlJfRt3MLDs9ryTOlh6OuqyINh4qFhCO7NZz8J7jidejYH567Fu46Ct6fxbhhBayvaM2b5d3CTikigbSwA0gj1743jH0K3nkmMtfU5DP47iGncI8N466dI1g45U2y0lLJSk8hKz2VzPTgeVoqWcHzzErvf/O68rJUstJSSEvVv41EakvFQsJnBr1OjYyYmvM3Ul77M09lPscr5f3Zuao1pRXG7gr7+mc5KZSRwlZS+JzI6wpSKPfIzwpSKK+8PPiJpZCamkZKaiopqWmkpqaSlppGatqe5+mkpqWRlpZKWloa6WnppKWlBc/TSE+P/MxITyctLZ2M9DQyMoKf6elkBo+MjHQyM9LJTE/DUtJI9TIc++bajDvgVX5SzTLHvQKvcByocI889wrAqaiowN2p2LNuueMErysqIutTEXyUU1EReZ89n+nBA775nj2f7Q4VTgWRdQi2/awsC4D3FswMdqW6c4XfXrb3KhX7f7+aj9xrkVfs932v5hymV1prQ2k2YPjr0/daL9FsKG1KqsV/5gNLlj4ChYWFXlxcHHYMqQ9b1/P5HUU0rfiSjJy2UFEe+eXg5fie5xXl4N88t+p+w4g0Eisq8ug14e1abWtmC929MNp6OrKQhqd5J9anRWZ76fPz17/1VnWThgGRf55WKSB4+TeFplLB+ea9imqW7Vnvm88oLytjd1kppaVl7C4ro7S0lNKyMkpLyygrizwvKyunrKyUsrKyyKO8nLKyMirKy9j6wUIco1nXfpGjKLPInphhez0neB6cMrMUIPKeAZZiOMH7RrBNSvCRhu1Zd8/3BJ+TUvV7gu8wA6t2+TfZIj9SSDFjwyv3Yhjth4+P8ofy7YVf79s+tqn6EV/v/35E+8zqLsnu2eSTmXcA0HHET6N+T0P3ycw7SEmroFecv0fFQpKDWWSkVRz+SqcCTYJHbSz7w1EA9LlsQn1FCs2yOX8GoM/wM0JOUkezbgWgd9FJIQepOw/2Jd50xU9ERKJSsRARkahULEREJCoVCxERiUrFQkREolKxEBGRqFQsREQkKhULERGJSsVCRESiUrEQEZGoVCxERCQqFQsREYlKxUJERKJSsRARkahULEREJCoVCxERiUrFQkREolKxEBGRqFQsREQkKhULERGJSsVCRESiUrEQEZGo4loszGyUma00s1Vmdt1+1jvbzNzMCoPXBWa2w8wWB4+74plTGp4+HVvQp2OLsGOISCAtXh9sZqnAROBEoARYYGbT3X15lfVygB8D86p8xPvuPiBe+aSBu+SZsBOISCXxPLIYDKxy99XuvhuYApxezXo3A/8F7IxjFhERqYN4FovOwLpKr0uCZV8zs4FAV3d/uprtu5nZm2b2ipkdHcecIiISRdxOQwFWzTL/+k2zFOB/gYurWe9jIM/dN5nZIOBfZtbH3bd+6wvMxgPjAfLy8uort4iIVGHuHn2t2nywWRHwe3cfGby+HsDdbwtetwDeB7YHm3QANgOnuXtxlc96GfhF1eWVFRYWenHxPt8WEZFqmNlCdy+Mtl48T0MtAHqYWTczywDOA6bvedPdv3D3tu5e4O4FwFyCQmFmucEFcszsIKAHsDqOWUVEZD/idhrK3cvM7CpgBpAK3O/uy8xsAlDs7tP3s/kxwAQzKwPKgSvcfXO8soqIyP7F7TTUgabTUCIiNdcQTkOJiEiSULEQEZGoVCxERCQqFQsREYlKxUJERKJSsRARkaiSZuismW0E1tbhI9oCn9VTnDAly36A9qWhSpZ9SZb9gLrtS76750ZbKWmKRV2ZWXEsY40bumTZD9C+NFTJsi/Jsh9wYPZFp6FERCQqFQsREYlKxeIb94QdoJ4ky36A9qWhSpZ9SZb9gAOwL7pmISIiUenIQkREolKxCJjZzWb2lpktNrOZZtYp7Ey1ZWZ/MrN3gv150sxahp2ptsxstJktM7MKM0u4kStmNsrMVprZKjO7Luw8dWFm95vZp2a2NOwsdWFmXc1slpmtCP5u/STsTLVlZllmNt/MlgT7clPcvkunoSLMrPmetq1m9mOgt7tfEXKsWjGzEcBLQU+RPwK4+69CjlUrZtYLqADuJkq3xIYmaOD1LnAikR70C4Dz3X15qMFqycyOIdLZ8iF37xt2ntoys45AR3dfZGY5wELgjET8czEzA5q6+3YzSwdeB37i7nPr+7t0ZBGo0t+7KZX6hScad5/p7mXBy7lAlzDz1IW7r3D3lWHnqKXBwCp3X+3uu4EpwOkhZ6o1d3+VSOvjhObuH7v7ouD5NmAF0DncVLXjEXtaU6cHj7j87lKxqMTMbjWzdcCFwA1h56knlwLPhR2ikeoMrKv0uoQE/aWUrMysABgIzAs3Se2ZWaqZLQY+Bf7j7nHZl0ZVLMzsBTNbWs3jdAB3/427dwUeBq4KN+3+RduXYJ3fAGVE9qfBimVfEpRVsyxhj1iTjZk1A54AflrlzEJCcfdydx9A5AzCYDOLyynCuPXgbojc/YQYV30EeAa4MY5x6iTavpjZOOBU4Hhv4BemavDnkmhKgK6VXncB1oeURSoJzu8/ATzs7v8MO099cPctZvYyMAqo90EIjerIYn/MrEell6cB74SVpa7MbBTwK+A0d/8q7DyN2AKgh5l1M7MM4DxgesiZGr3govDfgRXu/uew89SFmeXuGe1oZk2AE4jT7y6NhgqY2RPAIURG3qwFrnD3j8JNVTtmtgrIBDYFi+Ym8MiuM4G/ArnAFmCxu48MN1XszOxk4A4gFbjf3W8NOVKtmdmjwHAiM5xuAG5097+HGqoWzOwo4DXgbSL/vwP82t2fDS9V7ZhZP2ASkb9fKcBUd58Ql+9SsRARkWh0GkpERKJSsRARkahULEREJCoVCxERiUrFQkREolKxEKkBM9sefa39bj/NzA4Knjczs7vN7P1gxtBXzWyImWUEzxvVTbPSsKlYiBwgZtYHSHX31cGi+4hMzNfD3fsAFwNtg0kHXwTODSWoSDVULERqwSL+FMxh9baZnRssTzGz/wuOFJ42s2fN7OxgswuBp4L1ugNDgN+6ewVAMDvtM8G6/wrWF2kQdJgrUjvfAwYA/Ync0bzAzF4FjgQKgMOAdkSmv74/2OZI4NHgeR8id6OX7+PzlwJHxCW5SC3oyEKkdo4CHg1m/NwAvELkl/tRwOPuXuHunwCzKm3TEdgYy4cHRWR30JxHJHQqFiK1U9304/tbDrADyAqeLwP6m9n+/h/MBHbWIptIvVOxEKmdV4Fzg8YzucAxwHwibS3PCq5dtCcy8d4eK4CDAdz9faAYuCmYBRUz67Gnh4eZtQE2unvpgdohkf1RsRCpnSeBt4AlwEvAL4PTTk8Q6WOxlEjf8HnAF8E2z/Dt4vEDoAOwyszeBu7lm34XxwEJNwuqJC/NOitSz8ysmbtvD44O5gNHuvsnQb+BWcHrfV3Y3vMZ/wSuT+D+45JkNBpKpP49HTSkyQBuDo44cPcdZnYjkT7cH+5r46BR0r9UKKQh0ZGFiIhEpWsWIiISlYqFiIhEpWIhIiJRqViIiEhUKhYiIhKVioWIiET1/wFYUjoXMFo1RQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot CV误差曲线\n",
    "test_means = grid.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grid.cv_results_[ 'std_test_score' ]\n",
    "train_means = grid.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = grid.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "\n",
    "# plot results\n",
    "n_Cs = len(Cs)\n",
    "number_penaltys = len(penaltys)\n",
    "test_scores = np.array(test_means).reshape(n_Cs,number_penaltys)\n",
    "train_scores = np.array(train_means).reshape(n_Cs,number_penaltys)\n",
    "test_stds = np.array(test_stds).reshape(n_Cs,number_penaltys)\n",
    "train_stds = np.array(train_stds).reshape(n_Cs,number_penaltys)\n",
    "\n",
    "x_axis = np.log10(Cs)\n",
    "for i, value in enumerate(penaltys):\n",
    "    #pyplot.plot(log(Cs), test_scores[i], label= 'penalty:'   + str(value))\n",
    "    plt.errorbar(x_axis, -test_scores[:,i], yerr=test_stds[:,i] ,label = penaltys[i] +' Test')\n",
    "    #plt.errorbar(x_axis, -train_scores[:,i], yerr=train_stds[:,i] ,label = penaltys[i] +' Train')\n",
    "    \n",
    "plt.legend()\n",
    "plt.xlabel( 'log(C)' )                                                                                                      \n",
    "plt.ylabel( 'logloss' )\n",
    "plt.savefig('LogisticGridSearchCV_C.png' )\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 保存log似然损失的模型，用于后续测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "import _pickle as cPickle\n",
    "\n",
    "cPickle.dump(grid.best_estimator_, open(\"log_loss.pkl\", 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
